#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import scrapy
import time
#from gostudy.items import GostudyItem

class ChinaSpider(scrapy.Spider):
    name = 'china'
    start_urls = ['http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/index.html']

    def parse(self, response):
        item = []
        for inx, province in enumerate(response.css('tr.provincetr').css('td a')):
            # province_item = GostudyItem()
            # province_item['id'] = province.css('*::attr("href")').get().split('.')[0]
            # province_item['deep'] = 0
            # province_item['pid'] = 0
            # province_item['name'] = province.css('*::text').get()
            province_item = [province.css('*::attr("href")').get().split('.')[0], 0, 0, province.css('*::text').get()]
            item.append(province_item)
            yield scrapy.Request(response.urljoin(province.css('*::attr("href")').get()),
                                 callback=self.parse_two,errback=self.err_http_back_city, cb_kwargs=dict(parent_province=province_item))
        yield {'save_data': item}


    #获取城市
    def parse_two(self, response, parent_province):

        city_item = []
        try:
            for city in response.css('tr.citytr'):
                city_item_now = [city.css('td a::text')[0].get(), 1, parent_province[0], city.css('td a::text')[1].get()]
                city_item.append(city_item_now)
                yield scrapy.Request(response.urljoin(city.css('*::attr("href")')[0].get()), callback=self.parse_three, errback=self.err_http_back_county, cb_kwargs=dict(parent_city=city_item_now))
            yield {'save_data': city_item}
        except:
            fail_item = [response.url, parent_province, 2]
            yield {'save_data': fail_item, 'table': 'fail'}


    #获取县
    def parse_three(self, response, parent_city):
        county_item = []
        #此位置经常爬取不到数据  ，我们存储爬取不到数据,等待另外爬虫单独爬
        try:
            for county in response.css('tr.countytr'):
                # 第一行有点不一样
                if county.xpath('td/a/text()').get() is None:
                    county_brigt = [county.css('td::text')[0].get(), 2, parent_city[0],
                                    county.css('td::text')[1].get()]
                else:
                    county_brigt = [county.css('td a::text')[0].get(), 2, parent_city[0],
                                    county.css('td a::text')[1].get()]
                county_item.append(county_brigt)
            yield {'save_data': county_item}
        except:
            fail_item = [response.url, parent_city, 1]
            yield {'save_data': fail_item, 'table': 'fail'}
    def err_http_back_city(self, failure):
        request = failure.request
        fail_item = [request.url, request.cb_kwargs['parent_province'], 2]
        yield {'save_data': fail_item, 'table': 'fail'}
    def err_http_back_county(self, failure):
        request = failure.request
        fail_item = [request.url, request.cb_kwargs['parent_city'], 1]
        yield {'save_data': fail_item, 'table': 'fail'}





